{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sV_nvXSQGJPK"
   },
   "source": [
    "# Ungraded Lab: Using a multi-layer LSTM for forecasting\n",
    "\n",
    "In this lab, you will use the same RNN architecure in the first lab but will instead stack [LSTM](https://www.tensorflow.org/api_docs/python/tf/keras/layers/LSTM) layers instead of `SimpleRNN`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3IMbAWETGOWD"
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BOjujz601HcS"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kD7RQQ0xGQVH"
   },
   "source": [
    "## Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zswl7jRtGzkk"
   },
   "outputs": [],
   "source": [
    "def plot_series(time, series, format=\"-\", start=0, end=None):\n",
    "    \"\"\"\n",
    "    Visualizes time series data\n",
    "\n",
    "    Args:\n",
    "      time (array of int) - contains the time steps\n",
    "      series (array of int) - contains the measurements for each time step\n",
    "      format - line style when plotting the graph\n",
    "      start - first time step to plot\n",
    "      end - last time step to plot\n",
    "    \"\"\"\n",
    "\n",
    "    # Setup dimensions of the graph figure\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    \n",
    "    if type(series) is tuple:\n",
    "\n",
    "      for series_num in series:\n",
    "        # Plot the time series data\n",
    "        plt.plot(time[start:end], series_num[start:end], format)\n",
    "\n",
    "    else:\n",
    "      # Plot the time series data\n",
    "      plt.plot(time[start:end], series[start:end], format)\n",
    "\n",
    "    # Label the x-axis\n",
    "    plt.xlabel(\"Time\")\n",
    "\n",
    "    # Label the y-axis\n",
    "    plt.ylabel(\"Value\")\n",
    "\n",
    "    # Overlay a grid on the graph\n",
    "    plt.grid(True)\n",
    "\n",
    "    # Draw the graph on screen\n",
    "    plt.show()\n",
    "\n",
    "def trend(time, slope=0):\n",
    "    \"\"\"\n",
    "    Generates synthetic data that follows a straight line given a slope value.\n",
    "\n",
    "    Args:\n",
    "      time (array of int) - contains the time steps\n",
    "      slope (float) - determines the direction and steepness of the line\n",
    "\n",
    "    Returns:\n",
    "      series (array of float) - measurements that follow a straight line\n",
    "    \"\"\"\n",
    "\n",
    "    # Compute the linear series given the slope\n",
    "    series = slope * time\n",
    "\n",
    "    return series\n",
    "\n",
    "def seasonal_pattern(season_time):\n",
    "    \"\"\"\n",
    "    Just an arbitrary pattern, you can change it if you wish\n",
    "    \n",
    "    Args:\n",
    "      season_time (array of float) - contains the measurements per time step\n",
    "\n",
    "    Returns:\n",
    "      data_pattern (array of float) -  contains revised measurement values according \n",
    "                                  to the defined pattern\n",
    "    \"\"\"\n",
    "\n",
    "    # Generate the values using an arbitrary pattern\n",
    "    data_pattern = np.where(season_time < 0.4,\n",
    "                    np.cos(season_time * 2 * np.pi),\n",
    "                    1 / np.exp(3 * season_time))\n",
    "    \n",
    "    return data_pattern\n",
    "\n",
    "def seasonality(time, period, amplitude=1, phase=0):\n",
    "    \"\"\"\n",
    "    Repeats the same pattern at each period\n",
    "\n",
    "    Args:\n",
    "      time (array of int) - contains the time steps\n",
    "      period (int) - number of time steps before the pattern repeats\n",
    "      amplitude (int) - peak measured value in a period\n",
    "      phase (int) - number of time steps to shift the measured values\n",
    "\n",
    "    Returns:\n",
    "      data_pattern (array of float) - seasonal data scaled by the defined amplitude\n",
    "    \"\"\"\n",
    "    \n",
    "    # Define the measured values per period\n",
    "    season_time = ((time + phase) % period) / period\n",
    "\n",
    "    # Generates the seasonal data scaled by the defined amplitude\n",
    "    data_pattern = amplitude * seasonal_pattern(season_time)\n",
    "\n",
    "    return data_pattern\n",
    "\n",
    "def noise(time, noise_level=1, seed=None):\n",
    "    \"\"\"Generates a normally distributed noisy signal\n",
    "\n",
    "    Args:\n",
    "      time (array of int) - contains the time steps\n",
    "      noise_level (float) - scaling factor for the generated signal\n",
    "      seed (int) - number generator seed for repeatability\n",
    "\n",
    "    Returns:\n",
    "      noise (array of float) - the noisy signal\n",
    "    \"\"\"\n",
    "\n",
    "    # Initialize the random number generator\n",
    "    rnd = np.random.RandomState(seed)\n",
    "\n",
    "    # Generate a random number for each time step and scale by the noise level\n",
    "    noise = rnd.randn(len(time)) * noise_level\n",
    "    \n",
    "    return noise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yxNyeFWjGSdj"
   },
   "source": [
    "## Generate the Synthetic Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KYEUfDbdpHPm"
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "time = np.arange(4 * 365 + 1, dtype=\"float32\")\n",
    "baseline = 10\n",
    "amplitude = 40\n",
    "slope = 0.05\n",
    "noise_level = 5\n",
    "\n",
    "# Create the series\n",
    "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n",
    "\n",
    "# Update with noise\n",
    "series += noise(time, noise_level, seed=42)\n",
    "\n",
    "# Plot the results\n",
    "plot_series(time, series)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MYKCvv-eGWRc"
   },
   "source": [
    "## Split the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hpp0slenpKVD"
   },
   "outputs": [],
   "source": [
    "# Define the split time\n",
    "split_time = 1000\n",
    "\n",
    "# Get the train set \n",
    "time_train = time[:split_time]\n",
    "x_train = series[:split_time]\n",
    "\n",
    "# Get the validation set\n",
    "time_valid = time[split_time:]\n",
    "x_valid = series[split_time:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cktHz9aOGYtV"
   },
   "source": [
    "## Prepare Features and Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "G_j_2_Mqwn7-"
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "window_size = 20\n",
    "batch_size = 32\n",
    "shuffle_buffer_size = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4sTTIOCbyShY"
   },
   "outputs": [],
   "source": [
    "def windowed_dataset(series, window_size, batch_size, shuffle_buffer):\n",
    "    \"\"\"Generates dataset windows\n",
    "\n",
    "    Args:\n",
    "      series (array of float) - contains the values of the time series\n",
    "      window_size (int) - the number of time steps to include in the feature\n",
    "      batch_size (int) - the batch size\n",
    "      shuffle_buffer(int) - buffer size to use for the shuffle method\n",
    "\n",
    "    Returns:\n",
    "      dataset (TF Dataset) - TF Dataset containing time windows\n",
    "    \"\"\"\n",
    "\n",
    "    # Add an axis for the feature dimension of RNN layers\n",
    "    series = tf.expand_dims(series, axis=-1)\n",
    "\n",
    "    # Generate a TF Dataset from the series values\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(series)\n",
    "    \n",
    "    # Window the data but only take those with the specified size\n",
    "    dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n",
    "    \n",
    "    # Flatten the windows by putting its elements in a single batch\n",
    "    dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n",
    "\n",
    "    # Create tuples with features and labels \n",
    "    dataset = dataset.map(lambda window: (window[:-1], window[-1]))\n",
    "\n",
    "    # Shuffle the windows\n",
    "    dataset = dataset.shuffle(shuffle_buffer)\n",
    "    \n",
    "    # Create batches of windows\n",
    "    dataset = dataset.batch(batch_size)\n",
    "    \n",
    "    # Optimize the dataset for training\n",
    "    dataset = dataset.cache().prefetch(1)\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vGj5-InSwtQQ"
   },
   "outputs": [],
   "source": [
    "# Generate the dataset windows\n",
    "dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p9-Ke8ZaGcjd"
   },
   "source": [
    "## Build the Model\n",
    "\n",
    "As mentioned, you will swap `SimpleRNN` for `LSTM` in this lab. It is also set as bidirectional below but feel free to revise later and see what results you get. LSTMs are much more complex in their internal architecture than simpleRNNs. It implements a cell state that allows it to remember sequences better than simple implementations. This added complexity results in a bigger set of parameters to train and you'll see that when you print the model summary below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "A1Hl39rklkLm"
   },
   "outputs": [],
   "source": [
    "# Build the Model\n",
    "model_tune = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=(window_size, 1)),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n",
    "    tf.keras.layers.Dense(1),\n",
    "    tf.keras.layers.Lambda(lambda x: x * 100.0)\n",
    "])\n",
    "\n",
    "# Print the model summary\n",
    "model_tune.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FHaFblbwq0GV"
   },
   "source": [
    "## Tune the Learning Rate\n",
    "\n",
    "As usual, you will pick a learning rate by running the tuning code below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qE7al18qw48G"
   },
   "outputs": [],
   "source": [
    "# Set the learning rate scheduler\n",
    "lr_schedule = tf.keras.callbacks.LearningRateScheduler(\n",
    "    lambda epoch: 1e-8 * 10**(epoch / 20))\n",
    "\n",
    "# Initialize the optimizer\n",
    "optimizer = tf.keras.optimizers.SGD(momentum=0.9)\n",
    "\n",
    "# Set the training parameters\n",
    "model_tune.compile(loss=tf.keras.losses.Huber(), optimizer=optimizer)\n",
    "\n",
    "# Train the model\n",
    "history = model_tune.fit(dataset, epochs=100, callbacks=[lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AkBsrsXMzoWR"
   },
   "outputs": [],
   "source": [
    "# Define the learning rate array\n",
    "lrs = 1e-8 * (10 ** (np.arange(100) / 20))\n",
    "\n",
    "# Set the figure size\n",
    "plt.figure(figsize=(10, 6))\n",
    "\n",
    "# Set the grid\n",
    "plt.grid(True)\n",
    "\n",
    "# Plot the loss in log scale\n",
    "plt.semilogx(lrs, history.history[\"loss\"])\n",
    "\n",
    "# Increase the tickmarks size\n",
    "plt.tick_params('both', length=10, width=1, which='both')\n",
    "\n",
    "# Set the plot boundaries\n",
    "plt.axis([1e-8, 1e-3, 0, 30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i-lKNp4pq_w8"
   },
   "source": [
    "## Train the Model\n",
    "\n",
    "You can then proceed to train the model with your chosen learning rate. \n",
    "\n",
    "*Tip: When experimenting and you find yourself running different iterations of a model, you may want to use the [`clear_session()`](https://www.tensorflow.org/api_docs/python/tf/keras/backend/clear_session) method to declutter memory used by Keras. This is added in the first line below.*\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4uh-97bpLZCA"
   },
   "outputs": [],
   "source": [
    "# Reset states generated by Keras\n",
    "tf.keras.backend.clear_session()\n",
    "\n",
    "# Build the model\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=(window_size, 1)),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n",
    "    tf.keras.layers.Dense(1),\n",
    "    tf.keras.layers.Lambda(lambda x: x * 100.0)\n",
    "])\n",
    "\n",
    "# Set the learning rate\n",
    "learning_rate = 2e-6\n",
    "\n",
    "# Set the optimizer \n",
    "optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate, momentum=0.9)\n",
    "\n",
    "# Set the training parameters\n",
    "model.compile(loss=tf.keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "\n",
    "# Train the model\n",
    "history = model.fit(dataset,epochs=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RjP4kbGiI7cw"
   },
   "source": [
    "## Model Prediction\n",
    "\n",
    "You will then generate batches of windows to generate predictions that align with the validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QO8cfKwKBBG6"
   },
   "outputs": [],
   "source": [
    "def model_forecast(model, series, window_size, batch_size):\n",
    "    \"\"\"Uses an input model to generate predictions on data windows\n",
    "\n",
    "    Args:\n",
    "      model (TF Keras Model) - model that accepts data windows\n",
    "      series (array of float) - contains the values of the time series\n",
    "      window_size (int) - the number of time steps to include in the window\n",
    "      batch_size (int) - the batch size\n",
    "\n",
    "    Returns:\n",
    "      forecast (numpy array) - array containing predictions\n",
    "    \"\"\"\n",
    "\n",
    "    # Add an axis for the feature dimension of RNN layers\n",
    "    series = tf.expand_dims(series, axis=-1)\n",
    "    \n",
    "    # Generate a TF Dataset from the series values\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(series)\n",
    "\n",
    "    # Window the data but only take those with the specified size\n",
    "    dataset = dataset.window(window_size, shift=1, drop_remainder=True)\n",
    "\n",
    "    # Flatten the windows by putting its elements in a single batch\n",
    "    dataset = dataset.flat_map(lambda w: w.batch(window_size))\n",
    "    \n",
    "    # Create batches of windows\n",
    "    dataset = dataset.batch(batch_size).prefetch(1)\n",
    "    \n",
    "    # Get predictions on the entire dataset\n",
    "    forecast = model.predict(dataset, verbose=0)\n",
    "    \n",
    "    return forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_plB3UseBD8o"
   },
   "outputs": [],
   "source": [
    "# Reduce the original series\n",
    "forecast_series = series[split_time-window_size:-1]\n",
    "\n",
    "# Use helper function to generate predictions\n",
    "forecast = model_forecast(model, forecast_series, window_size, batch_size)\n",
    "\n",
    "# Drop single dimensional axis\n",
    "results = forecast.squeeze()\n",
    "\n",
    "# Plot the results\n",
    "plot_series(time_valid, (x_valid, results))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Nn8iSQkZtaRC"
   },
   "source": [
    "You can then generate the metrics to evaluate the model's performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-IKhueZaBGID"
   },
   "outputs": [],
   "source": [
    "# Compute the MSE and MAE\n",
    "print(tf.keras.metrics.mse(x_valid, results).numpy())\n",
    "print(tf.keras.metrics.mae(x_valid, results).numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bzPIqeMWvbPy"
   },
   "source": [
    "## Wrap Up\n",
    "\n",
    "This concludes this short exercise on using LSTMs for time series forecasting. Next week, you will build upon this and add convolutions. Then, you will start to move away from synthetic data and use real-world datasets. See you there!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "U5PqTePGHeMH"
   },
   "source": [
    "## Optional: Including a Validation Set while Training\n",
    "\n",
    "Back in the first course of this specialization, you saw how you can also monitor the performance of your model against a validation set while training. You can also do that for this lab. \n",
    "\n",
    "First, you need to generate a `val_set` which are data windows and labels that your model can accept. You can simply reuse the `windowed_dataset` function for that and you can pass in the `x_valid` points to generate the windows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kvFVwL3PC4iX"
   },
   "outputs": [],
   "source": [
    "# Generate data windows of the validation set\n",
    "val_set = windowed_dataset(x_valid, window_size, batch_size, shuffle_buffer_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p68j6XxEvP9K"
   },
   "source": [
    "You can then do the same training as before but pass in the `val_set` to the `validation_data` parameter of the `fit()` method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wujKz6tXDSn0"
   },
   "outputs": [],
   "source": [
    "# Reset states generated by Keras\n",
    "tf.keras.backend.clear_session()\n",
    "\n",
    "# Build the model\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=(window_size, 1)),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n",
    "    tf.keras.layers.Dense(1),\n",
    "    tf.keras.layers.Lambda(lambda x: x * 100.0)\n",
    "])\n",
    "\n",
    "# Set the learning rate\n",
    "learning_rate = 2e-6\n",
    "\n",
    "# Set the optimizer \n",
    "optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate, momentum=0.9)\n",
    "\n",
    "# Set the training parameters\n",
    "model.compile(loss=tf.keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "\n",
    "# Train the model\n",
    "history = model.fit(dataset,epochs=100, validation_data=val_set)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "C4_W3_Lab_2_LSTM.ipynb",
   "private_outputs": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
